'''
Created on Feb 2, 2010

@author: oabalbin
'''

#import sys;
#import string;
#import StringIO;
import os;
import sys;
import glob;
import time
import MySQLdb;
import numpy as np
from optparse import OptionParser
from datetime import datetime
from datetime import date

# My libraries
import signatures.parsers.read_microarray_data as pm
import signatures.parsers.read_gene_lists as gpm
import signatures.db.tables as dt
import signatures.db.query as dq

import signatures.bfrm.output_parser as bo

class pipeline():
    
    def __init__(self, dbhost, dbuser, dbpasswd, database):
        
        self.mt = dt.tables(dbhost, dbuser, dbpasswd, database)
        self.mq = dq.query(dbhost, dbuser, dbpasswd, database)
        #table information
        self.tableDescription = 'probeID VARCHAR(40), geneSym VARCHAR(40), sample VARCHAR(40), log_fold FLOAT(20,14), INDEX probe (probeID), INDEX gene (geneSym), INDEX sample (sample)'
        self.tableName = 'NCI60'
        
        # Parameters for pre-processing data:
        # percentage of low variable gene to be filter out of the analysis
        self.lvar=25
        # percentage of low median gene to be filter out of the analysis
        self.lmed=25
        
        # Parameters to parse brfm output. 
        # inter= intercept (true or false), covar= number of covariates, nresfac=number of response factors, nlatfac= number of latent factors, 
        # ppith =  threshold. It determines a threshold for the probability of gene i to be included in factor j.
        self.inter, self.covar, self.nresfac, self.nlatfac, self.ppith = 0, 0 ,0, 0, 0.99 
    
    
    def parse_input_annotfiles(self, annotInputFile):
    
        inputdirs, seed_file, outdirs, typefile={},{},{},'nofile'
    
        for line in annotInputFile:
            fields = line.strip('\n')
                 
            if fields[0][0] == '':
                continue
            
            if fields[0][0] == '#':
                typefile = fields
                continue
            fields = fields.split('=')
            dict = {'#Input Directories':inputdirs, '#Seed genes':seed_file,'#Output Directories':outdirs}[typefile]
            
            if typefile == '#Input Directories':
                dict[fields[0]] = fields[1]
            else:
                dict[fields[0]] = fields[1]
            
        return inputdirs, seed_file, outdirs
    

    def read_files_folder(self,folderpath,ext):
        ''' '''
        # Read files in folder
        myfiles=[]
        for infile in glob.glob( os.path.join(folderpath, '*'+ext) ):
            myfiles.append(infile)
            #print "current file is: " + infile
        return myfiles
    
    
    def process_microarray_files(self,exp_files,tp):
        dbf=[]
        for f in exp_files:
            of=f+'_dbfile'
            tp.microarray_nci60_file(open(f),open(of,'w'))
            dbf.append(of)
        
        return dbf
    
    def create_database(self,dbfiles,doit=False):
        """
        It creates a databases if doit = True
        """
        if doit:
            print 'Making tables'            
            for dbf in dbfiles:     
                if not(self.mt.table_existence(self.tableName)):
                    self.mt.create_table(self.tableName,self.tableDescription)
                    
                self.mt.load_data_table(self.tableName,dbf)
                
    def check_create_dir( self, full_path_name ):
        """
        creates an output folder time stamped
        """
        t = datetime.now()
        outfolder = full_path_name + t.strftime("%Y_%m_%d_%H_%M")
        if not os.path.isdir(outfolder):
            os.mkdir(outfolder)
        
        return outfolder
    
    def check_bfrm_output(self, bfrmfolder):
        """
        Check if files  mA.txt, mF.txt, mPostPib.txt, exist
        """
        date = bfrmfolder.split('/')[-3]
        genefile = date+'_dataset.gen.txt'
        bfrmof = ['mA.txt', 'mPostPib.txt', 'mF.txt',genefile]
        files=[]   
        
        for f in bfrmof:
            fname = bfrmfolder + f
           
            if os.path.isfile(fname):
                files.append(fname)
                
        return files

     
    def write_pathway_activity(self,pathactivity,samplelist, outfolder):
        
        avgf = open(outfolder+'avg_sign.txt','w')
        medf = open(outfolder+'med_sign.txt','w')
        stdf = open(outfolder+'std_sign.txt','w')
        
        avgf.write('Factor'+'\t'+",".join(samplelist).replace(",", "\t") + '\n')
        medf.write('Factor'+'\t'+",".join(samplelist).replace(",", "\t") + '\n')
        stdf.write('Factor'+'\t'+",".join(samplelist).replace(",", "\t") + '\n')
        
        
        for i, fact in pathactivity.iteritems():
            avgf.write(str(i+1)+'\t'+",".join(map(str,fact[0])).replace(",", "\t") + '\n')
            medf.write(str(i+1)+'\t'+",".join(map(str,fact[1])).replace(",", "\t") + '\n')
            stdf.write(str(i+1)+'\t'+",".join(map(str,fact[2])).replace(",", "\t") + '\n')

##########

if __name__ == '__main__':
    
    optionparser = OptionParser("usage: %prog [options] ")
    optionparser.add_option("-f", "--annotFile", dest="annotFile",
                            help="annotation file for all files to use")
    optionparser.add_option("-d", action="store_true", dest="dbtrue",
                            help="create a database. Bool")

    
    (options, args) = optionparser.parse_args()
       
    # start the pipeline. 
    # open the database
    pipe = pipeline("localhost", "oabalbin", "oscar", "signatures")
    inputdirs, seed_file, outdirs = pipe.parse_input_annotfiles(open(options.annotFile))    
    #    
    outfile1 = outdirs['outdir']+'allFactors_matrix.txt'  # Factors
    
    ######### Get factors data  ############
    tp = pm.parser()
    
    ## dirs diferent directories where data is
    mastergenelist = tp.list_of_names(open(seed_file['univfile']))
    
    allFactors = np.zeros((len(mastergenelist),1))
    alllabels=[] 
    
    for d, thisdir in inputdirs.iteritems():
        
        print thisdir
        
        bfof = pipe.check_bfrm_output(thisdir)
        
        exp_genlist = tp.list_of_names(open(bfof[3]))
        
        if len(bfof) < 4:
            print 'Not all BFRM output files were found'
            sys.exit(0)
        else:
            mA = np.loadtxt(bfof[0],dtype=float, delimiter='\t')
            nvar, nfactors = mA.shape[0], mA.shape[1]
            exp_genlist = tp.list_of_names(open(bfof[3]))
        
        bfrmparser = bo.bfrm_parser(pipe.inter, pipe.covar, pipe.nresfac, pipe.nlatfac, pipe.ppith, nvar, nfactors)
        # it returns a dictionary of the final genes, and thir ind, present in all factors
        newA = bfrmparser.get_gene_in_thismodule(bfof[1], bfof[0], exp_genlist, mastergenelist,True)
        lab = range(newA.shape[1])
        for h in lab:
            th = d+'_'+str(h)
            alllabels.append(th)
         
        allFactors = np.hstack((allFactors,newA))
    
    print allFactors
    print alllabels
    
    outfile = open(outfile1,'w')
    alllabels = map(str,alllabels)
    outfile.write('Gene'+'\t'+'0'+'\t'+",".join(alllabels).replace(',','\t')+'\n')
    for i in range(allFactors.shape[0]):
        g = mastergenelist[i]
        if np.flatnonzero(allFactors[i,:]).shape[0] == 0:
            print 'Zero element'
            continue
        el = map(str,list(allFactors[i,:]))
        outfile.write(g+'\t'+",".join(el).replace(',','\t')+'\n')
    
    sys.exit(0)
    
    
    
    